﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections.Concurrent;
using Crawler_v1.utils;
using CsQuery;
using Crawler_v1.spider.entity;
using Crawler_v1.entity;

namespace Crawler_v1.spider
{
    class Schedule
    {
        public object locker = new object();//锁


        private ConcurrentQueue<Page> pages = new ConcurrentQueue<Page>();

        //public static string BASE_URL = "http://fec.mofcom.gov.cn";

        private string columnUrl;//栏目url
        private string columnName;//栏目名称
        private string parent;//父栏目
        private string listUrl;
        private int page = 1;

        public int Page
        {
            get { return page; }
            set { page = value; }
        }

        public Schedule(string column_url,string column_name,string parent)
        {
            this.columnUrl = column_url;
            this.columnName = column_name;
            this.parent = parent;
        }


        private ThreadSpider spider;
        public void onAttachSpider(ThreadSpider spider)
        {
            this.spider = spider;
        }


        public Page getOneUrl()
        {
            Page outPage;

            if (page == 1)
            {
                listUrl = Constants.BASE_URL + columnUrl;//  http://fec.mofcom.gov.cn/article/fwydyl/tjsj/
            }
            else
            {
                listUrl = Constants.BASE_URL + columnUrl + "?" + page;
            }

            if (pages.TryDequeue(out outPage))
            {

            }
            else
            {
                //获取
                string html = HttpUtil.GetHtml(listUrl);

                CQ dom = html;
                var section = dom["section.newList_01"];

                var list = dom["section.newList_01 ul li"].Each((index, eachDom) =>
                {
                    var li = CQ.Create(eachDom);

                    var a = li.Children("a");
                    var span = li.Children("span");
                    if (a.Length > 0 && span.Length > 0)
                    {
                        var href = a.Attr<string>("href");
                        var dateString = span.Text();

                        //创建新的
                        Page page1 = new Page();
                        page1.Url = Constants.BASE_URL + href;
                        Article article = new Article();

                        DateTime dateTime = Convert.ToDateTime(dateString);
                        //DateTime dateTime = DateTime.ParseExact(dateString, "yyyy-MM-dd hh:mm:ss", System.Globalization.CultureInfo.CurrentCulture);
                        article.date = dateTime;
                        article.column = columnName;
                        article.parent = parent;

                        page1.Data = article;


                        pages.Enqueue(page1);

                    }

                });

                if (pages.Count<=0)
                {
                    spider.SpiderListener.onMsg("列表页已遍历完成，没有文章，当前页数:" + page);
                }
                else
                {
                    page++;//下一页
                }

                /*
                foreach (var ele in list)
                {
                    string href = ele.Attributes["href"];

                    //创建新的
                    Page page1 = new Page();
                    page1.Url = url;
                    Article article = new Article();
                    DateTime.ParseExact(dateString, "yyyyMMdd", System.Globalization.CultureInfo.CurrentCulture);
                    article.date = new System.DateTime();

                    urls.Enqueue(href);
                }
                */

                pages.TryDequeue(out outPage);

                
            };



            //page1.Data=new 
            return outPage;//为null的话spider也会停止
        }

    }
}
